import requests
from lxml import etree
import time
import sqlite3
import re


class Zgx(object):
    def __init__(self, name, username, s_id):
        self.name = name
        self.username = username
        self.s_id = s_id

        self.conn = sqlite3.connect('F:/单子项目/project_41910_微博分析系统/WeboStom/db.sqlite3')  # sqlite3的地址
        self.cursor = self.conn.cursor()

    def req(self, url):
        response = requests.get(url)
        response.encoding = 'utf-8'
        text = response.text
        time.sleep(0.5)
        return text

    def spider(self):
        url = "http://www.chinanews.com/scroll-news/news1.html"
        text = self.req(url)
        html = etree.HTML(text)
        ul = html.xpath('//div[@class="content_list"]/ul/li')
        for li in ul:
            types = li.xpath('.//div[@class="dd_lm"]/a/text()')
            if not types:
                continue

            types = types[0]  # 类型

            url = li.xpath('.//div[@class="dd_bt"]/a/@href')
            if not url:
                continue

            url = "http:" + url[0]  # url

            text = self.req(url)
            html = etree.HTML(text)
            content = ''.join(html.xpath('//p/text()'))
            if not content:
                continue

            item = {
                "types": types,
                "content": content
            }

            self.save(item)

    def save(self, item):
        try:
            tables = "systaam_xinlanspider"

            sql = "INSERT INTO {} (name, author, content, move, comment, good, spider_id, user_id, classify) VALUES " \
                  "('{}', '{}', '{}',{}, {}, {}, {}, '{}', '{}');".format(tables,
                                                                          self.name, "中国新闻", item['content'],
                                                                          0, 0,
                                                                          0,
                                                                          self.s_id, self.username, item['types']
                                                                          )

            self.cursor.execute(sql)
            self.conn.commit()

        except Exception as e:
            print(e)
            return


def zg_spider(name, username, s_id):
    a = Zgx(name, username, s_id)
    a.spider()


class ZgSwWeb(object):
    def __init__(self, name, username, s_id, key):
        self.name = name
        self.username = username
        self.s_id = s_id
        self.key = key

        self.conn = sqlite3.connect('F:/单子项目/project_41910_微博分析系统/WeboStom/db.sqlite3')  # sqlite3的地址
        self.cursor = self.conn.cursor()

        self.urls = ("http://search.dangdang.com/?key=%s&page_index=%d" % (key, i) for i in range(1, 4))

    def spider(self):
        for url in self.urls:
            response = requests.get(url)
            html = etree.HTML(response.text)

            ul = html.xpath('//ul[@class="bigimg"]/li')
            for li in ul:
                info = li.xpath('./p[@class="detail"]/text()')  # 详细
                if len(info) > 0:
                    info = info[0]  # 说明
                    item = {
                        'types': self.key,
                        'content': info
                    }

                    self.save(item)

    def save(self, item):
        try:
            tables = "systaam_xinlanspider"

            sql = "INSERT INTO {} (name, author, content, move, comment, good, spider_id, user_id, classify) VALUES " \
                  "('{}', '{}', '{}',{}, {}, {}, {}, '{}', '{}');".format(tables,
                                                                          self.name, "单单网", item['content'],
                                                                          0, 0,
                                                                          0,
                                                                          self.s_id, self.username, item['types']
                                                                          )

            self.cursor.execute(sql)
            self.conn.commit()

        except Exception as e:
            print(e)
            return


def dan_spider(name, username, s_id, key):
    a = ZgSwWeb(name, username, s_id, key)
    a.spider()


